{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Format DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(442, 11)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>bmi</th>\n",
       "      <th>bp</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>s6</th>\n",
       "      <th>progression</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.038076</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>0.061696</td>\n",
       "      <td>0.021872</td>\n",
       "      <td>-0.044223</td>\n",
       "      <td>-0.034821</td>\n",
       "      <td>-0.043401</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>0.019908</td>\n",
       "      <td>-0.017646</td>\n",
       "      <td>151.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.001882</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.051474</td>\n",
       "      <td>-0.026328</td>\n",
       "      <td>-0.008449</td>\n",
       "      <td>-0.019163</td>\n",
       "      <td>0.074412</td>\n",
       "      <td>-0.039493</td>\n",
       "      <td>-0.068330</td>\n",
       "      <td>-0.092204</td>\n",
       "      <td>75.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.085299</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>0.044451</td>\n",
       "      <td>-0.005671</td>\n",
       "      <td>-0.045599</td>\n",
       "      <td>-0.034194</td>\n",
       "      <td>-0.032356</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>0.002864</td>\n",
       "      <td>-0.025930</td>\n",
       "      <td>141.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.089063</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.011595</td>\n",
       "      <td>-0.036656</td>\n",
       "      <td>0.012191</td>\n",
       "      <td>0.024991</td>\n",
       "      <td>-0.036038</td>\n",
       "      <td>0.034309</td>\n",
       "      <td>0.022692</td>\n",
       "      <td>-0.009362</td>\n",
       "      <td>206.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.005383</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.036385</td>\n",
       "      <td>0.021872</td>\n",
       "      <td>0.003935</td>\n",
       "      <td>0.015596</td>\n",
       "      <td>0.008142</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>-0.031991</td>\n",
       "      <td>-0.046641</td>\n",
       "      <td>135.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        age       sex       bmi        bp        s1        s2        s3  \\\n",
       "0  0.038076  0.050680  0.061696  0.021872 -0.044223 -0.034821 -0.043401   \n",
       "1 -0.001882 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163  0.074412   \n",
       "2  0.085299  0.050680  0.044451 -0.005671 -0.045599 -0.034194 -0.032356   \n",
       "3 -0.089063 -0.044642 -0.011595 -0.036656  0.012191  0.024991 -0.036038   \n",
       "4  0.005383 -0.044642 -0.036385  0.021872  0.003935  0.015596  0.008142   \n",
       "\n",
       "         s4        s5        s6  progression  \n",
       "0 -0.002592  0.019908 -0.017646        151.0  \n",
       "1 -0.039493 -0.068330 -0.092204         75.0  \n",
       "2 -0.002592  0.002864 -0.025930        141.0  \n",
       "3  0.034309  0.022692 -0.009362        206.0  \n",
       "4 -0.002592 -0.031991 -0.046641        135.0  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.datasets import load_diabetes\n",
    "\n",
    "data = load_diabetes()\n",
    "train_df = pd.DataFrame(data=data.data, columns=data.feature_names)\n",
    "train_df[\"progression\"] = data.target\n",
    "\n",
    "print(train_df.shape)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set Up Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Experiment Key:   'SM0fBtJM_PadIDgPuyTT0v3Vweau04D4XNhWpqJ1BEY='\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import Environment, CVExperiment\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "\n",
    "env = Environment(\n",
    "    train_dataset=train_df,\n",
    "    results_path=\"HyperparameterHunterAssets\",\n",
    "    target_column=\"progression\",\n",
    "    metrics=dict(msle=(mean_squared_log_error, \"min\")),\n",
    "    cv_type=\"KFold\",\n",
    "    cv_params=dict(n_splits=10, random_state=42),\n",
    "    runs=3,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that HyperparameterHunter has an active `Environment`, we can do two things:\n",
    "\n",
    "# 1. Perform Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<20:54:34> Validated Environment:  'SM0fBtJM_PadIDgPuyTT0v3Vweau04D4XNhWpqJ1BEY='\n",
      "<20:54:34> Initialized Experiment: '7ac42393-1e8e-4b62-a6b2-4abcc2569fc2'\n",
      "<20:54:34> Hyperparameter Key:     'M898kSqnvKDNsKDrekegUv9zMnuV1E57RHWNXNe7_00='\n",
      "<20:54:34> \n",
      "<20:54:34> F0/R0  |  OOF(msle=0.22658)  |  Time Elapsed: 0.19622 s\n",
      "<20:54:34> F0/R1  |  OOF(msle=0.22658)  |  Time Elapsed: 0.19427 s\n",
      "<20:54:35> F0/R2  |  OOF(msle=0.22658)  |  Time Elapsed: 0.19377 s\n",
      "<20:54:35> F0.0 AVG:   OOF(msle=0.22658)  |  Time Elapsed: 0.59082 s\n",
      "<20:54:35> F1/R0  |  OOF(msle=0.33111)  |  Time Elapsed: 0.1958 s\n",
      "<20:54:35> F1/R1  |  OOF(msle=0.33111)  |  Time Elapsed: 0.19596 s\n",
      "<20:54:35> F1/R2  |  OOF(msle=0.33111)  |  Time Elapsed: 0.19581 s\n",
      "<20:54:35> F0.1 AVG:   OOF(msle=0.33111)  |  Time Elapsed: 0.5942 s\n",
      "<20:54:35> F2/R0  |  OOF(msle=0.19371)  |  Time Elapsed: 0.19539 s\n",
      "<20:54:36> F2/R1  |  OOF(msle=0.19371)  |  Time Elapsed: 0.19476 s\n",
      "<20:54:36> F2/R2  |  OOF(msle=0.19371)  |  Time Elapsed: 0.19474 s\n",
      "<20:54:36> F0.2 AVG:   OOF(msle=0.19371)  |  Time Elapsed: 0.59165 s\n",
      "<20:54:36> F3/R0  |  OOF(msle=0.18992)  |  Time Elapsed: 0.19469 s\n",
      "<20:54:36> F3/R1  |  OOF(msle=0.18992)  |  Time Elapsed: 0.1952 s\n",
      "<20:54:36> F3/R2  |  OOF(msle=0.18992)  |  Time Elapsed: 0.19507 s\n",
      "<20:54:36> F0.3 AVG:   OOF(msle=0.18992)  |  Time Elapsed: 0.5921 s\n",
      "<20:54:37> F4/R0  |  OOF(msle=0.22097)  |  Time Elapsed: 0.20227 s\n",
      "<20:54:37> F4/R1  |  OOF(msle=0.22097)  |  Time Elapsed: 0.19729 s\n",
      "<20:54:37> F4/R2  |  OOF(msle=0.22097)  |  Time Elapsed: 0.19703 s\n",
      "<20:54:37> F0.4 AVG:   OOF(msle=0.22097)  |  Time Elapsed: 0.60347 s\n",
      "<20:54:37> F5/R0  |  OOF(msle=0.27765)  |  Time Elapsed: 0.20319 s\n",
      "<20:54:37> F5/R1  |  OOF(msle=0.27765)  |  Time Elapsed: 0.19495 s\n",
      "<20:54:38> F5/R2  |  OOF(msle=0.27765)  |  Time Elapsed: 0.19557 s\n",
      "<20:54:38> F0.5 AVG:   OOF(msle=0.27765)  |  Time Elapsed: 0.60015 s\n",
      "<20:54:38> F6/R0  |  OOF(msle=0.23880)  |  Time Elapsed: 0.19476 s\n",
      "<20:54:38> F6/R1  |  OOF(msle=0.23880)  |  Time Elapsed: 0.19429 s\n",
      "<20:54:38> F6/R2  |  OOF(msle=0.23880)  |  Time Elapsed: 0.1946 s\n",
      "<20:54:38> F0.6 AVG:   OOF(msle=0.23880)  |  Time Elapsed: 0.59065 s\n",
      "<20:54:38> F7/R0  |  OOF(msle=0.19713)  |  Time Elapsed: 0.19561 s\n",
      "<20:54:38> F7/R1  |  OOF(msle=0.19713)  |  Time Elapsed: 0.19565 s\n",
      "<20:54:39> F7/R2  |  OOF(msle=0.19713)  |  Time Elapsed: 0.19533 s\n",
      "<20:54:39> F0.7 AVG:   OOF(msle=0.19713)  |  Time Elapsed: 0.59346 s\n",
      "<20:54:39> F8/R0  |  OOF(msle=0.24813)  |  Time Elapsed: 0.19501 s\n",
      "<20:54:39> F8/R1  |  OOF(msle=0.24813)  |  Time Elapsed: 0.1955 s\n",
      "<20:54:39> F8/R2  |  OOF(msle=0.24813)  |  Time Elapsed: 0.1957 s\n",
      "<20:54:39> F0.8 AVG:   OOF(msle=0.24813)  |  Time Elapsed: 0.59305 s\n",
      "<20:54:39> F9/R0  |  OOF(msle=0.19174)  |  Time Elapsed: 0.19623 s\n",
      "<20:54:40> F9/R1  |  OOF(msle=0.19174)  |  Time Elapsed: 0.20027 s\n",
      "<20:54:40> F9/R2  |  OOF(msle=0.19174)  |  Time Elapsed: 0.19602 s\n",
      "<20:54:40> F0.9 AVG:   OOF(msle=0.19174)  |  Time Elapsed: 0.59961 s\n",
      "<20:54:40> \n",
      "<20:54:40> FINAL:    OOF(msle=0.23179)  |  Time Elapsed: 5.95923 s\n",
      "<20:54:40> \n",
      "<20:54:40> Saving results for Experiment: '7ac42393-1e8e-4b62-a6b2-4abcc2569fc2'\n"
     ]
    }
   ],
   "source": [
    "from rgf import RGFRegressor\n",
    "\n",
    "experiment = CVExperiment(\n",
    "    model_initializer=RGFRegressor,\n",
    "    model_init_params=dict(max_leaf=2000, algorithm='RGF', min_samples_leaf=10),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Hyperparameter Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validated Environment with key: \"SM0fBtJM_PadIDgPuyTT0v3Vweau04D4XNhWpqJ1BEY=\"\n",
      "\u001b[31mSaved Result Files\u001b[0m\n",
      "\u001b[31m______________________________________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   algorithm |        l2 |   learning_rate |      loss |   normalize | \n",
      "Experiments matching cross-experiment key/algorithm: 1\n",
      "Experiments fitting in the given space: 1\n",
      "Experiments matching current guidelines: 1\n",
      "    0 | 7ac42393 | 00m00s | \u001b[35m   0.23179\u001b[0m | \u001b[32m        RGF\u001b[0m | \u001b[32m   0.1000\u001b[0m | \u001b[32m         0.5000\u001b[0m | \u001b[32m       LS\u001b[0m | \u001b[32m          1\u001b[0m | \n",
      "\u001b[31mHyperparameter Optimization\u001b[0m\n",
      "\u001b[31m______________________________________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   algorithm |        l2 |   learning_rate |      loss |   normalize | \n",
      "    1 | 5ac47409 | 00m17s |   21.74256 |     RGF_Sib |    0.1710 |          0.5947 |      Expo |           0 | \n",
      "    2 | 1929e9e4 | 00m08s | \u001b[35m   0.22132\u001b[0m | \u001b[32m    RGF_Sib\u001b[0m | \u001b[32m   0.2793\u001b[0m | \u001b[32m         0.5386\u001b[0m | \u001b[32m       LS\u001b[0m | \u001b[32m          0\u001b[0m | \n",
      "    3 | 9b60c7de | 00m01s |    0.32706 |         RGF |    0.2022 |          0.5206 |      Expo |           1 | \n",
      "    4 | 21b0ae9b | 00m17s |   21.32219 |     RGF_Sib |    0.0245 |          0.6065 |      Expo |           0 | \n",
      "    5 | b2513cdb | 00m09s | \u001b[35m   0.21850\u001b[0m | \u001b[32m    RGF_Sib\u001b[0m | \u001b[32m   0.2717\u001b[0m | \u001b[32m         0.4808\u001b[0m | \u001b[32m       LS\u001b[0m | \u001b[32m          1\u001b[0m | \n",
      "    6 | b699aa67 | 00m09s |    0.22587 |     RGF_Sib |    0.2157 |          0.6243 |        LS |           1 | \n",
      "    7 | 6af34f1c | 00m01s |    0.32706 |         RGF |    0.0583 |          0.3180 |      Expo |           1 | \n",
      "    8 | 76e08515 | 00m20s |   21.92036 |         RGF |    0.2369 |          0.4316 |       Log |           0 | \n",
      "    9 | ef1de97a | 00m01s |    0.32706 |     RGF_Opt |    0.0966 |          0.6939 |      Expo |           1 | \n",
      "   10 | fb964ca4 | 00m17s |   21.85549 |     RGF_Sib |    0.2859 |          0.6878 |      Expo |           0 | \n",
      "   11 | 2f5e48dc | 00m12s |    0.23609 |     RGF_Opt |    0.0426 |          0.6100 |        LS |           0 | \n",
      "   12 | 4cbdec64 | 00m06s |    0.22889 |         RGF |    0.1629 |          0.5876 |        LS |           0 | \n",
      "   13 | c4c6e9d9 | 00m09s |    0.23739 |     RGF_Sib |    0.1321 |          0.6702 |        LS |           1 | \n",
      "   14 | e93351d7 | 00m09s |    0.22431 |     RGF_Sib |    0.1939 |          0.5232 |        LS |           0 | \n",
      "   15 | 553a46c2 | 00m09s |    0.22117 |     RGF_Sib |    0.2998 |          0.6506 |        LS |           0 | \n",
      "   16 | 4987f8b5 | 00m09s |    0.22646 |     RGF_Sib |    0.2348 |          0.6801 |        LS |           1 | \n",
      "   17 | 43cbec27 | 00m06s |    0.22319 |         RGF |    0.2713 |          0.3418 |        LS |           1 | \n",
      "   18 | e1c3168f | 00m09s |    0.22870 |     RGF_Sib |    0.2559 |          0.6842 |        LS |           1 | \n",
      "   19 | 887037b4 | 00m09s |    0.23245 |     RGF_Sib |    0.1401 |          0.4059 |        LS |           0 | \n",
      "   20 | bbadc1d5 | 00m12s |    0.22975 |     RGF_Opt |    0.2644 |          0.3197 |        LS |           1 | \n",
      "   21 | 8b90a5ba | 00m09s |    0.21997 |     RGF_Sib |    0.1954 |          0.4344 |        LS |           0 | \n",
      "   22 | 05d8969f | 00m09s |    0.22618 |     RGF_Sib |    0.1742 |          0.3497 |        LS |           0 | \n",
      "   23 | ef91f72f | 00m09s |    0.22548 |     RGF_Sib |    0.1789 |          0.5327 |        LS |           0 | \n",
      "   24 | fb57617c | 00m09s |    0.23316 |     RGF_Sib |    0.2469 |          0.4210 |        LS |           1 | \n",
      "   25 | fe5f890d | 00m09s |    0.22186 |     RGF_Sib |    0.2852 |          0.4937 |        LS |           1 | \n",
      "   26 | 43a6d91d | 00m09s |    0.22478 |     RGF_Sib |    0.2715 |          0.3854 |        LS |           1 | \n",
      "   27 | 802c417d | 00m09s | \u001b[35m   0.21629\u001b[0m | \u001b[32m    RGF_Sib\u001b[0m | \u001b[32m   0.2615\u001b[0m | \u001b[32m         0.4825\u001b[0m | \u001b[32m       LS\u001b[0m | \u001b[32m          1\u001b[0m | \n",
      "   28 | 7f4d5807 | 00m09s |    0.21768 |     RGF_Sib |    0.2601 |          0.4725 |        LS |           1 | \n",
      "   29 | 8c8ec1ec | 00m09s |    0.21910 |     RGF_Sib |    0.2586 |          0.4678 |        LS |           1 | \n",
      "   30 | 9ea95fa6 | 00m09s |    0.22000 |     RGF_Sib |    0.2379 |          0.4766 |        LS |           1 | \n",
      "Optimization loop completed in 0:04:56.452760\n",
      "Best score was 0.21629200355319095 from Experiment \"802c417d-74fa-4dcd-959b-7948557870f4\"\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import ExtraTreesOptPro, Real, Integer, Categorical\n",
    "\n",
    "optimizer = ExtraTreesOptPro(iterations=30, random_state=42)\n",
    "\n",
    "optimizer.forge_experiment(\n",
    "    model_initializer=RGFRegressor,\n",
    "    model_init_params=dict(\n",
    "        max_leaf=2000,\n",
    "        algorithm=Categorical(['RGF', 'RGF_Opt', 'RGF_Sib']),\n",
    "        l2=Real(0.01, 0.3),\n",
    "        normalize=Categorical([True, False]),\n",
    "        learning_rate=Real(0.3, 0.7),\n",
    "        loss=Categorical(['LS', 'Expo', 'Log']),\n",
    "    ),\n",
    ")\n",
    "\n",
    "optimizer.go()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice, `optimizer` recognizes our earlier `experiment`'s hyperparameters fit inside the search space/guidelines set for `optimizer`.\n",
    "\n",
    "Then, when optimization is started, it automatically learns from `experiment`'s results - without any extra work for us!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}